
package com.kinth.work.zggys;

import java.util.*;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.PriorityScheduler;

/**
 * 
 * <p>
 * Title: 获得总的列表ip.
 * </p>
 * <p>
 * Description: 首先是https://cn.china.cn/company/,这是总的起始页。而GetBigUrl是处理这一页的。
 * 用来得到下一页的链接。这里有个问题，就是此页得到的链接是不能用的。也就是直接利用links()函数得到的链接是不能用
 * 的。如果用这个链接，在地址栏会自动改变地址。我也不知道为什么。为了解决这个问题，需要自己解析拼接地址。经过观察，
 * （也就是2里面的地址),你可以发现，只需要把w=破译掉即可。而它正式这个链接的text()的中文的Url编码。所以，直接 自己提取出来自己组合即可。
 * </p>
 * 
 * @author kaikai
 * @2017年4月1日
 */
public class GetBigUrl implements PageProcessor
{
	private Site site = Site.me().setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "
			+ "(KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36").setRetryTimes(3).setSleepTime(100);

	public static final String BIG_URL = "https://cn.china.cn/company/";
	public  static final String BIG_URL2 = "https://www.china.cn/\\w+/\\?t=101";
	public  static final String BIG_URL3 = "https://www.china.cn/\\w+/\\?t=101&p=\\d+";
	public  static final String COMPANY_URL = "https://[a-zA-Z0-9]+.cn.china.cn/";
	

    
	@Override
	public void process(Page page)
	{
		
		if(page.getUrl().toString().equals(BIG_URL))
		{
			processBigUrl(page);
		}else if(page.getUrl().regex(BIG_URL2).match())
		{
			processBigUrl2(page);
		}else if(page.getUrl().regex(BIG_URL3).match())
		{
			processBigUrl3(page);
		}else if(page.getUrl().regex(COMPANY_URL).match())
		{
			getInfo(page);
		}

	}

	//处理公司页面
	public void getInfo(Page page)
	{

		String name = page.getHtml()
				.xpath("//body/div[@class='content-wrapper']" + "/div[@class='bd']/div[@class='content fl-clr']"
						+ "/div[@class='con-lef']/div[@id='det_info']/div[@class='lef-bd']"
						+ "/ul/li[@class='comp']/h3/a/text()")
				.toString();
		if (name == null)
		{

			name = page
					.getHtml().xpath("//body" + "//div[@class='content fl-clr']"
							+ "//div[@id='det_info']/div[@class='lef-bd']" + "/ul//div[@class='comp']/a/text()")
					.toString();

		}
		String people = null;
		String address = null;
		String phone1 = null;
		String phone2 = null;
		String phone3 = null;
		String qq = null;
		String introduction = null;
		String product = null;

		if (name != null)
		{
			// 产品
			product = page.getHtml().xpath("//head/title/text()").toString();

			// 产品介绍
			introduction = page.getHtml().xpath("//body" + "//div[@class='content fl-clr']"
					+ "//div[@id='det_info']/div[@class='lef-bd']" + "/ul/li[5]").toString();
			if (introduction == null)
			{
				introduction = page.getHtml()
						.xpath("//body" + "//div[@class='content fl-clr']" + "//div[@class='company-info']/p/text()")
						.toString();
			}
			//替换取出提取的字符串里面的html代码
			String regex;
			regex = "<[\\s\\S]*?>";
			introduction = introduction.replaceAll(regex, "").trim();
			introduction = introduction.replaceAll("\n", "");

			// 对联系方式里面的利用循环处理
			int amount = 7;
			for (int i = 1; i <= amount; i++)
			{
				String tempKey = page
						.getHtml().xpath("//body" + "//div[@class='content fl-clr']"
								+ "//div[@id='contact']/div[@class='lef-bd']" + "/ul/li[" + i + "]/dl/dt/text()")
						.toString();
				String tempValue = page
						.getHtml().xpath("//body" + "//div[@class='content fl-clr']"
								+ "//div[@id='contact']/div[@class='lef-bd']" + "/ul/li[" + i + "]/dl/dd/text()")
						.toString();
				if (tempKey == null)
				{
					break;
				}
				if (tempKey.substring(0, 1).equals("联"))
				{
					people = tempValue;

				} else if (tempKey.substring(0, 1).equals("职"))
				{
					String people2 = tempValue;

					// 联系人加职位
					people = people + " " + people2;

				} else if (tempKey.substring(0, 1).equals("电"))
				{
					phone1 = tempKey + tempValue;

				} else if (tempKey.substring(0, 1).equals("手"))
				{
					phone2 = tempKey + tempValue;

				} else if (tempKey.substring(0, 1).equals("传"))
				{
					phone3 = tempKey + tempValue;

				} else if (tempKey.substring(0, 1).equals("地"))
				{
					address = tempKey + tempValue;

				}
				if (tempKey.substring(0, 1).equals("客"))
				{
					qq = page.getHtml()
							.xpath("//body" + "//div[@class='content fl-clr']"
									+ "//div[@id='contact']/div[@class='lef-bd']" + "/ul/li[" + i + "]/dl/dd/a")
							.toString();
					if (qq != null)
					{
						String qqTemp[] = qq.split("=");
						if (qqTemp.length >= 2)
						{
							qq = qqTemp[1];
							qqTemp = qq.split("\\(");
							if (qqTemp.length >= 2)
							{
								qq = qqTemp[1];
								qqTemp = qq.split(",");
								if (qqTemp.length >= 2)
								{
									qq = qqTemp[0];
								} else
								{
									qq = null;
								}
							} else
							{
								qq = null;
							}
						} else
						{
							qq = null;
						}
					}
				}

			}

		}

		String urlNow = page.getUrl().toString();
		String industry = page.getRequest().getExtra("industry").toString();
		page.putField("name", name);
		page.putField("product", product);
		page.putField("address", address);
		page.putField("people", people);
		page.putField("qq", qq);
		page.putField("introduction", introduction);
		page.putField("phone1", phone1);
		page.putField("phone2", phone2);
		page.putField("phone3", phone3);
		page.putField("industry", industry);
		page.putField("url", urlNow);

	}
	//处理列表页
	public void processBigUrl3(Page page)
	{
		ArrayList<String> companyList = new ArrayList<String>();
		companyList = (ArrayList<String>) page.getHtml()
				.xpath("//body/div[@class='s-layout']" + "/div[@class='s-mod-main']/div[@class='cont-left']"
						+ "/div[@class='col']/div[@class='contbox']//h3/a")
				.links().all();
		
		
		String industry = page.getRequest().getExtra("industry").toString();
		
		
		for (String url : companyList)
		{
			System.out.println("urlnext: "+url);
			System.out.println("industry: "+page.getRequest().getExtra("industry"));
			Request request = new Request(url).setPriority(2).putExtra("industry", industry);
	        try
			{
				Thread.sleep(1000);
			} catch (InterruptedException e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			page.addTargetRequest(request); 
		}
	}
	//处理列表页首页
	public void processBigUrl2(Page page)
	{
		String pageAmount = page.getHtml()
				.xpath("/html/body//div[@class='content']" 
		        + "/div[@class='location_an']/span/b/text()")
				.toString();
        //System.out.println("总页数:url:  "+urlNow+","+pageAmount);
        
		if (pageAmount != null)
		{
			String urlNow = page.getUrl().toString();
			int pageAmount2 = Integer.parseInt(pageAmount);
			int pageAmount3 = pageAmount2 / 20 + 1;

			if (pageAmount3 > 30)
			{
				pageAmount3 = 30;
			}
			for (int i = 1; i <= pageAmount3; i++)
			{
				String urlnext = urlNow + "&p=" + i;
				System.out.println("urlnext: "+urlnext);
				System.out.println("industry: "+page.getRequest().getExtra("industry"));
				Request request = new Request(urlnext).setPriority(1).putExtra("industry", page.getRequest().getExtra("industry"));
		        try
				{
					Thread.sleep(1000);
				} catch (InterruptedException e)
				{
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
				page.addTargetRequest(request); 
			}
		}
	}
	//处理首页
	public void processBigUrl(Page page)
	{
		ArrayList<String> nameList = new ArrayList<String>();
		ArrayList<String> urlList = new ArrayList<String>();
		
		urlList = (ArrayList<String>) page.getHtml()
				.xpath("//body/div[@class='nav_line']" + "//div[@class='nav_pop']//a").links().all();
		nameList = (ArrayList<String>) page.getHtml()
				.xpath("//body/div[@class='nav_line']" + "//div[@class='nav_pop']//a/text()").all();
		
		 
		for (int i = 0; i < urlList.size(); i++)
		{
			String url = urlList.get(i);
			String name = nameList.get(i);
			System.out.println("urlnext: "+url);
			System.out.println("industry: "+name);
			Request request = new Request(url).setPriority(0).putExtra("industry", name);
	        try
			{
				Thread.sleep(1000);
			} catch (InterruptedException e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			page.addTargetRequest(request); 
			
		}
	}
	@Override
	public Site getSite()
	{
		return site;
	}

	// 一个代替main函数的函数
	public static void runSpider()
	{
		System.out.println("爬取类别链接开始");
		GetBigUrl bigUrl = new GetBigUrl();
		Spider.create(bigUrl)
		.setScheduler(new PriorityScheduler())
		.addPipeline(new DataPipeline())
		.addUrl("https://cn.china.cn/company/").run();

	}

	public static void main(String[] args)
	{
		runSpider();
	}

	
}